#!/bin/bash

THIS_NODE_IP="10.0.0.164"
ANOTHER_NODE_IP="10.0.0.165"
MASTER_NODE_IP="10.0.0.164"
BACKUP_NODE_IP="10.0.0.165"
VIP="10.0.0.166"

MESSAGE_SERVER="10.0.0.100:1234"
MESSAGE_FILE_START="gxcc_start.txt"
MESSAGE_FILE_STOP="gxcc_stop.txt"

WEB_DIR="/var/www/html"
LOCK_FILE_START="gxcc_start.locking"
LOCK_FILE_STOP="gxcc_stop.locking"
LOCK_FILE_CHECK="gxcc_check.locking"
LOCK_FILE_BACKUP="gxcc_backup.locking"

function check_this_node_ip()
{
    if (/usr/sbin/ip a | grep $THIS_NODE_IP) > /dev/null;then return 0;else return 1;fi
}

function check_connect_this_node()
{
    ip_array=("10.0.0.100" "10.0.0.101" "10.0.0.102")
    for ip in ${ip_array[@]};
    do
	if (ping -c 1 $ip) > /dev/null;then return 0;fi
    done
    return 1
}

function check_connect_another_node()
{
    if (ping -c 1 $1) > /dev/null;then return 0;else return 1;fi
}

function check_this_node_running()
{
    r=0
    echo "<p>=====检查本节点是否处于RUNNING状态：开始=====</p>" >> $1
    mount | grep " /data " | grep "dev" | grep "rw" > /dev/null
    if [ $? -eq 0 ];then
	echo "<p>本节点共享磁盘：挂载，读写模式</p>" >> $1
    else
	echo "<p>本节点共享磁盘：未挂载，或非读写模式</p>" >> $1
	r=1
    fi
    /bin/systemctl status nfs.service | grep "Active: active" > /dev/null
    if [ $? -eq 0 ];then
	echo "<p>本节点NFS服务：运行</p>" >> $1
    else
	echo "<p>本节点NFS服务：未运行</p>" >> $1
	r=1
    fi
    /bin/systemctl status vsftpd.service | grep "Active: active" > /dev/null
    if [ $? -eq 0 ];then
	echo "<p>本节点VSFTP服务：运行</p>" >> $1
    else
	echo "<p>本节点VSFTP服务：未运行</p>" >> $1
	r=1
    fi
    /usr/sbin/ip a | grep $VIP > /dev/null
    if [ $? -eq 0 ];then
	echo "<p>本节点虚拟IP：启用</p>" >> $1
    else
	echo "<p>本节点虚拟IP：未启用</p>" >> $1
	r=1
    fi
    check_connect_this_node
    if [ $? -eq 0 ];then
	echo "<p>本节点网络：可达</p>" >> $1
    else
	echo "<p>本节点网络：不可达</p>" >> $1
	r=1
    fi
    echo "<p>=====检查本节点是否处于RUNNING状态：结束=====</p>" >> $1
    return $r
}

function check_another_node_running()
{
    r=1
    echo "<p>=====检查另一节点是否处于RUNNING状态：开始=====</p>" >> $1
    rm -f /tmp/index.html > /dev/null
    wget -P /tmp/ http://$2/index.html > /dev/null
    if [ -f /tmp/index.html ];then
	line=`grep "STATE:RUNNING" /tmp/index.html`
	time=`echo $line | grep -P '20[0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]' -o`
	echo $line | grep "STATE:RUNNING" > /dev/null
	if [ $? -eq 0 ];then
	    echo "<p>在另一节点状态页面检测到RUNNING状态</p>" >> $1
	    echo $time | grep "20" > /dev/null
	    if [ $? -eq 0 ];then
		a=`date -d "$time" +%s`
		b=`date +%s`
		if [ $[ $b - $a ] -lt 65 ];then
		    echo "<p>状态页面更新时间在65秒内</p>" >> $1
		    r=0
		else
		    echo "<p>状态页面更新时间不在65秒内：${time}</p>" >> $1
		fi
	    else
		echo "<p>没有解析到更新时间</p>" >> $1
	    fi
	else
	    echo "<p>没有在另一节点状态页面检测到RUNNING状态</p>" >> $1
	fi
    else
	echo "<p>获取另一节点状态页面失败</p>" >> $1
    fi
    echo "<p>=====检查另一节点是否处于RUNNING状态：结束=====</p>" >> $1
    return $r
}

function check_this_node_stopped()
{
    r=0
    echo "<p>=====检查本节点是否处于STOPPED状态：开始=====</p>" >> $1
    mount | grep " /data " > /dev/null
    if [ $? -eq 0 ];then
	echo "<p>本节点共享磁盘：挂载</p>" >> $1
	r=1
    else
	echo "<p>本节点共享磁盘：未挂载</p>" >> $1
    fi
    /bin/systemctl status nfs.service | grep "Active: inactive" > /dev/null
    if [ $? -eq 0 ];then
	echo "<p>本节点NFS服务：未运行</p>" >> $1
    else
	echo "<p>本节点NFS服务：运行</p>" >> $1
	r=1
    fi
    /bin/systemctl status vsftpd.service | grep "Active: inactive" > /dev/null
    if [ $? -eq 0 ];then
	echo "<p>本节点VSFTP服务：未运行</p>" >> $1
    else
	echo "<p>本节点VSFTP服务：运行</p>" >> $1
	r=1
    fi
    /usr/sbin/ip a | grep $VIP > /dev/null
    if [ $? -eq 0 ];then
	echo "<p>本节点虚拟IP：启用</p>" >> $1
	r=1
    else
	echo "<p>本节点虚拟IP：未启用</p>" >> $1
    fi
    echo "<p>=====检查本节点是否处于STOPPED状态：结束=====</p>" >> $1
    return $r
}

function check_another_node_stopped()
{
    r=1
    echo "<p>=====检查另一节点是否处于STOPPED状态：开始=====</p>" >> $1
    rm -f /tmp/index.html > /dev/null
    wget -P /tmp/ http://$2/index.html > /dev/null
    if [ -f /tmp/index.html ];then
	line=`grep "STATE:STOPPED" /tmp/index.html`
	time=`echo $line | grep -P '20[0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]' -o`
	echo $line | grep "STATE:STOPPED" > /dev/null
	if [ $? -eq 0 ];then
	    echo "<p>在另一节点状态页面检测到STOPPED状态</p>" >> $1
	    a=`date -d "$time" +%s`
	    b=`date +%s`
	    if [ $[ $b - $a ] -lt 65 ];then
		echo "<p>状态页面更新时间在65秒内</p>" >> $1
		r=0
	    else
		echo "<p>状态页面更新时间不在65秒内：${time}</p>" >> $1
	    fi
	else
	    echo "<p>没有在另一节点状态页面检测到STOPPED状态</p>" >> $1
	fi
    else
	echo "<p>获取另一节点状态页面失败</p>" >> $1
    fi
    echo "<p>=====检查另一节点是否处于STOPPED状态：结束=====</p>" >> $1
    return $r
}

function check_another_node_locking()
{
    r=0
    echo "<p>=====检查另一节点是否处于START_LOCKING状态：开始=====</p>" >> $1
    rm -f /tmp/${LOCK_FILE_START} > /dev/null
    wget -P /tmp/ http://$2/${LOCK_FILE_START} > /dev/null
    if [ -f /tmp/${LOCK_FILE_START} ];then
	echo "<p>另一节点处于START_LOCKING状态</p>" >> $1
    else
	echo "<p>另一节点不是START_LOCKING状态</p>" >> $1
	r=1
    fi
    echo "<p>=====检查另一节点是否处于START_LOCKING状态：结束=====</p>" >> $1
    return $r
}

function check_this_node_before_start()
{
    r=0
    echo "<p>=====进行本节点启动前检查：开始=====</p>" >> $1
    mount | grep " /data " > /dev/null
    if [ $? -eq 0 ];then
	echo "<p>本节点共享磁盘：挂载</p>" >> $1
	r=1
    else
	echo "<p>本节点共享磁盘：未挂载</p>" >> $1
    fi
    /bin/systemctl status nfs.service | grep "Active: inactive" > /dev/null
    if [ $? -eq 0 ];then
	echo "<p>本节点NFS服务：未运行</p>" >> $1
    else
	echo "<p>本节点NFS服务：运行</p>" >> $1
	r=1
    fi
    /bin/systemctl status vsftpd.service | grep "Active: inactive" > /dev/null
    if [ $? -eq 0 ];then
	echo "<p>本节点VSFTP服务：未运行</p>" >> $1
    else
	echo "<p>本节点VSFTP服务：运行</p>" >> $1
	r=1
    fi
    /usr/sbin/ip a | grep $VIP > /dev/null
    if [ $? -eq 0 ];then
	echo "<p>本节点虚拟IP：启用</p>" >> $1
	r=1
    else
	echo "<p>本节点虚拟IP：未启用</p>" >> $1
    fi
    check_connect_this_node
    if [ $? -eq 0 ];then
	echo "<p>本节点网络：可达</p>" >> $1
    else
	echo "<p>本节点网络：不可达</p>" >> $1
	r=1
    fi
    ping -c 1 $VIP > /dev/null
    if [ $? -eq 0 ];then
	echo "<p>虚拟IP：能ping通</p>" >> $1
	r=1
    else
	echo "<p>虚拟IP：不能ping通</p>" >> $1
    fi
    echo "<p>=====进行本节点启动前检查：结束=====</p>" >> $1
    return $r
}

function start_this_node()
{
    r=0
    echo "<p>=====启动本节点：开始=====</p>" >> $1
    mount /dev/data/lvol0 /data/ > /dev/null
    mount | grep " /data " | grep "dev" | grep "rw" > /dev/null
    if [ $? -eq 0 ];then
	echo "<p>共享磁盘：挂载，读写模式</p>" >> $1
    else
	echo "<p>共享磁盘：未挂载，或非读写模式</p>" >> $1
	r=1
    fi
    /bin/systemctl start nfs.service > /dev/null
    /bin/systemctl status nfs.service | grep "Active: active" > /dev/null
    if [ $? -eq 0 ];then
	echo "<p>NFS服务：运行</p>" >> $1
    else
	echo "<p>NFS服务：未运行</p>" >> $1
	r=1
    fi
    /bin/systemctl start vsftpd.service > /dev/null
    /bin/systemctl status vsftpd.service | grep "Active: active" > /dev/null
    if [ $? -eq 0 ];then
	echo "<p>VSFTP服务：运行</p>" >> $1
    else
	echo "<p>VSFTP服务：未运行</p>" >> $1
	r=1
    fi
    /usr/sbin/ip addr add $VIP/24 dev ens160 > /dev/null
    /usr/sbin/ip a | grep $VIP > /dev/null
    if [ $? -eq 0 ];then
	echo "<p>本节点虚拟IP：启用</p>" >> $1
    else
	echo "<p>本节点虚拟IP：未启用</p>" >> $1
	r=1
    fi
    echo "<p>=====启动本节点：结束=====</p>" >> $1
    return $r
}

#该函数不应进行stop操作
#如果同时执行两个启动流程，后启动的可能会关闭前面已经启动的服务
function start_this_node_with_check()
{
    check_this_node_before_start $1
    if [ $? -eq 0 ];then
	echo "<p>本节点启动前检查通过</p>" >> $1
	start_this_node $1
	if [ $? -eq 0 ];then
	    return 0
	fi
    else
	echo "<p>本节点启动前检查不通过</p>" >> $1
    fi
    return 1
}

function stop_this_node()
{
    r=0
    echo "<p>=====停止本节点：开始=====</p>" >> $1
    /usr/sbin/ip addr del $VIP/24 dev ens160 > /dev/null
    /usr/sbin/ip a | grep $VIP > /dev/null
    if [ $? -eq 0 ];then
	echo "<p>本节点虚拟IP：停用失败</p>" >> $1
	r=1
    else
	echo "<p>本节点虚拟IP：已停用</p>" >> $1
    fi
    /bin/systemctl stop nfs.service > /dev/null
    /bin/systemctl status nfs.service | grep "Active: inactive" > /dev/null
    if [ $? -eq 0 ];then
	echo "<p>NFS服务：已停止</p>" >> $1
    else
	echo "<p>NFS服务：停止失败</p>" >> $1
	r=1
    fi
    /bin/systemctl stop vsftpd.service > /dev/null
    /bin/systemctl status vsftpd.service | grep "Active: inactive" > /dev/null
    if [ $? -eq 0 ];then
	echo "<p>VSFTP服务：已停止</p>" >> $1
    else
	echo "<p>VSFTP服务：停止失败</p>" >> $1
	r=1
    fi
    umount -l /data/ > /dev/null
    mount | grep " /data " > /dev/null
    if [ $? -eq 0 ];then
	echo "<p>共享磁盘：卸载失败</p>" >> $1
	r=1
    else
	echo "<p>共享磁盘：已卸载</p>" >> $1
    fi
    echo "<p>=====停止本节点：结束=====</p>" >> $1
    return $r
}

function check_start_message()
{
    rm -f /tmp/${MESSAGE_FILE_START} > /dev/null
    wget -P /tmp/ http://$MESSAGE_SERVER/${MESSAGE_FILE_START} > /dev/null
    if [ -f /tmp/${MESSAGE_FILE_START} ];then
	echo "<p>在${MESSAGE_SERVER}上获得${MESSAGE_FILE_START}，正在解析</p>" >> $1
	ip=`grep -P '10.0.0.16[4-5]' -o /tmp/${MESSAGE_FILE_START}`
	echo $ip | grep "10" > /dev/null
	if [ $? -eq 0 ];then
	    if [ "$ip"x = "$THIS_NODE_IP"x ];then
		echo "<p>2个节点都STOPPED时，将启动${THIS_NODE_IP}</p>" >> $1
		return 0
	    else
		echo "<p>没有解析到本节点IP地址：${ip}</p>" >> $1
	    fi
	else
	    echo "<p>没有解析到IP地址</p>" >> $1
	fi
    else
	echo "<p>在${MESSAGE_SERVER}上没有${MESSAGE_FILE_START}</p>" >> $1
	echo "<p>${MESSAGE_FILE_START}的内容应当只有一行，为优先启动的节点的ip地址</p>" >> $1
    fi
    return 1
}

function check_stop_message()
{
    rm -f /tmp/${MESSAGE_FILE_STOP} > /dev/null
    wget -P /tmp/ http://$MESSAGE_SERVER/${MESSAGE_FILE_STOP} > /dev/null
    if [ -f /tmp/${MESSAGE_FILE_STOP} ];then
	echo "<p>在${MESSAGE_SERVER}上获得${MESSAGE_FILE_STOP}，正在解析</p>" >> $1
	ip=`grep -P '10.0.0.16[4-5]' -o /tmp/${MESSAGE_FILE_STOP}`
	time=`grep -P '20[0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]' -o /tmp/${MESSAGE_FILE_STOP}`
	echo $ip | grep "10" > /dev/null
	if [ $? -eq 0 ];then
	    if [ "$ip"x = "$THIS_NODE_IP"x ];then
		echo $time | grep "20" > /dev/null
		if [ $? -eq 0 ];then
		    a=`date -d "$time" +%s`
		    b=`date +%s`
		    if [ $[ $b - $a ] -lt 300 ]&&[ $[ $b - $a ] -gt -300 ];then
			echo "<p>收到停止本节点服务指令，已在规定时间5分钟内</p>" >> $MYLOGFILE
			return 0
		    elif [ $[ $b - $a ] -gt 300 ];then
			echo "<p>收到停止本节点服务指令，但时间已经过了</p>" >> $MYLOGFILE
		    elif [ $[ $b - $a ] -lt -300 ];then
			c=`date -d @$a "+%Y-%m-%d %H:%M"`
			echo "<p>收到停止本节点服务指令，本节点将在${c}停服务</p>" >> $MYLOGFILE
		    else
			echo "<p>收到停止本节点服务指令，但时间错误：${time}</p>" >> $MYLOGFILE
		    fi
		else
		    echo "<p>没有解析到时间</p>" >> $1
		fi
	    else
		echo "<p>没有解析到本节点IP地址：${ip}</p>" >> $1
	    fi
	else
	    echo "<p>没有解析到IP地址</p>" >> $1
	fi
    else
	echo "<p>在${MESSAGE_SERVER}上没有${MESSAGE_FILE_STOP}，该文件格式应当包含如下两行：</p>" >> $1
	echo "<p>ip</p>" >> $1
	echo "<p>yyyy-mm-dd hh24:mi</p>" >> $1
    fi
    return 1
}

function check_httpd()
{
    /bin/systemctl status httpd.service | grep "Active: active" > /dev/null
    if [ $? -ne 0 ];then
	/bin/systemctl start httpd.service > /dev/null
    fi
}

function check_ntpd()
{
    /bin/systemctl status ntpd.service | grep "Active: active" > /dev/null
    if [ $? -ne 0 ];then
	/bin/systemctl start ntpd.service > /dev/null
    fi
}
